package com.ifengchao.util;

import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.ifengchao.dao.iface.DealReptileDao;
import com.ifengchao.pojo.HConfig;
import com.ifengchao.pojo.HCrawlLog;
import com.ifengchao.pojo.HMessagePlat;
import com.ifengchao.service.impl.*;
import com.ifengchao.util.fdfs.FastDFSFile;
import com.ifengchao.util.fdfs.FileManager;
import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.apache.log4j.Logger;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.*;
import java.net.URL;
import java.net.URLConnection;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import static com.ifengchao.util.DateUtils.getDayBegin;

/**
 * @Author YiMing
 * @Description:
 * @Date: Created in 上午10:55 2017/9/15
 * @Modified By
 */
public class ReptileUtil {

	private static final Logger logger = Logger.getLogger(ReptileUtil.class);

	/**
	 * Gets html data.
	 *
	 * @param uri the uri
	 * @return the html data
	 * @throws Exception the exception
	 */
	//httpClient 发送请求
	public static String getHtmlData(String uri) throws Exception {
		CloseableHttpClient client = HttpClients.createDefault();
		HttpGet post = new HttpGet(uri);
		post.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:50.0) Gecko/20100101 Firefox/50.0");
		CloseableHttpResponse resp = null;
		try {
			resp = client.execute(post);
		} catch (Exception e) {
			throw new Exception("网络与服务器连接异常！");
		}
		if (resp.getStatusLine().getStatusCode() == 200) {
			HttpEntity entity = resp.getEntity();
			return EntityUtils.toString(entity, "utf-8");

		} else {
			System.out.println("请求失败");
			return null;
		}
	}

	/**
	 * Encode string.
	 *
	 * @param str the str
	 * @return the string
	 * @throws Exception the exception
	 */
//解码
	public static String encode(String str) throws Exception {
		// 定义正则表达式来搜索中文字符的转义符号
		Pattern compile = Pattern.compile("&#.*?;");
		// 测试用中文字符
		Matcher matcher = compile.matcher(str);
		// 循环搜索 并转换 替换
		while (matcher.find()) {
			String group = matcher.group();
			// 获得16进制的码
			String hexcode = "0" + group.replaceAll("(&#|;)", "");
			// 字符串形式的16进制码转成int并转成char 并替换到源串中
			str = str.replaceAll(group, (char) Integer.decode(hexcode).intValue() + "");
		}
		return str;
	}

	/**
	 * Decode unicode string.
	 *
	 * @param dataStr 汉字转化为Unicode后的字符串
	 * @return String Unicode码转化为的汉字
	 * @throws
	 * @Title: decodeUnicode
	 * @Description: 将Unicode码转为汉字<br>
	 */
	public static String decodeUnicode(final String dataStr) {
		int start = 0;
		int end = 0;
		final StringBuffer buffer = new StringBuffer();
		while (start > -1) {
			end = dataStr.indexOf("\\u", start + 2);
			String charStr = "";
			if (end == -1) {
				charStr = dataStr.substring(start + 2, dataStr.length());
			} else {
				charStr = dataStr.substring(start + 2, end);
			}
			char letter = (char) Integer.parseInt(charStr, 16); // 16进制parse整形字符串。
			buffer.append(new Character(letter).toString());
			start = end;
		}
		return buffer.toString();
	}

	/**
	 * Upload string.
	 *
	 * @param url the url
	 * @return the string
	 * @throws Exception the exception
	 */
	//上传图片到fastdfs 全路径
	public static String upload(String hyzUrl, String url) throws Exception {

		File directory = new File("");// 参数为空
		String courseFile = directory.getCanonicalPath();
		courseFile += "/src/config";

		String ext = null;
		String fileName = null;
		int i = url.lastIndexOf(".");

		if (i == -1) {
			ext = "jpg";
		} else {
			ext = url.substring(i + 1, i + 4);
			if (!ext.startsWith("jpg") && !ext.startsWith("png") && !ext.startsWith("gif")) {
				ext = "jpg";
			}
		}
		fileName = "1." + ext;


		logger.info("开始下载网上图片到本地");

		try {
			download(url, fileName, courseFile);
		} catch (Exception e) {
			logger.error("===========下载失败!!图片地址是:" + url + "===========");
			return null;
		}

		logger.info("成功下载到本地 地址是:" + courseFile + "/" + fileName + ",开始上传FastDFS");

		//开始上传
		File content = new File(courseFile + "/" + fileName);

		logger.info("本地图片路径" + courseFile + "/" + fileName);

		FileInputStream fis = new FileInputStream(content);

		byte[] file_buff = null;
		if (fis != null) {
			int len = fis.available();
			file_buff = new byte[len];
			fis.read(file_buff);
		}

		FastDFSFile file = new FastDFSFile(fileName, file_buff, ext);

		String fileAbsolutePath = null;

		try {
			fileAbsolutePath = FileManager.upload(file);
		} catch (Exception e) {
			logger.info("=======图片从FastDFS上传失败=======");
			content.delete();//删除本地图片文件
			return null;
		}

		fis.close();//释放资源

		content.delete();//删除本地图片文件

		logger.info("上传成功,FastDFS的地址是:" + fileAbsolutePath);
		/*String replace = fileAbsolutePath.replace("/group1/", "");

		FileInfo group1 = FileManager.getFile("group1", replace);

		Assert.notNull(group1);
		String sourceIpAddr = group1.getSourceIpAddr();
		long size = group1.getFileSize();

		String path = "http://" + sourceIpAddr + fileAbsolutePath;
		logger.info("上传成功,FastDFS的地址是:" + path + "图片大小是:" + size);*/

		return hyzUrl + fileAbsolutePath;

	}

	/**
	 * Upload string.
	 *
	 * @param url the url
	 * @return the string
	 * @throws Exception the exception
	 */
	//上传图片到fastdfs 半路径
	public static String upload(String url) throws Exception {

		File directory = new File("");// 参数为空
		String courseFile = directory.getCanonicalPath();
		courseFile += "/src/config";

		String ext = null;
		String fileName = null;
		/*int i = url.lastIndexOf(".");

		if (i == -1) {
			ext = "jpg";
		} else {
			ext = url.substring(i + 1, i + 4);
			if (!ext.startsWith("jpg") && !ext.startsWith("png") && ext.startsWith("gif")) {
				ext = "jpg";
			}
		}*/

		fileName = "1.jpg";

		logger.info("开始下载网上图片到本地");

		try {
			download(url, fileName, courseFile);
		} catch (Exception e) {
			logger.error("===========下载失败!!图片地址是:" + url + "===========");
			return null;
		}

		logger.info("成功下载到本地 地址是:" + courseFile + "/" + fileName + ",开始上传FastDFS");

		//压缩图片
		ImageUtil.pictureAlter(courseFile + "/" + fileName, courseFile + "/" + "2.jpg");

		//开始上传
		File content = new File(courseFile + "/" + "2.jpg");

		logger.info("本地图片路径" + courseFile + "/" + fileName);

		FileInputStream fis = new FileInputStream(content);

		byte[] file_buff = null;
		if (fis != null) {
			int len = fis.available();
			file_buff = new byte[len];
			fis.read(file_buff);
		}

		FastDFSFile file = new FastDFSFile("2.jpg", file_buff, "jpg");

		String fileAbsolutePath = null;

		try {
			fileAbsolutePath = FileManager.upload(file);
		} catch (Exception e) {
			logger.info("=======图片从FastDFS上传失败=======");
			content.delete();//删除本地图片文件
			return null;
		}

		fis.close();//释放资源

		content.delete();//删除本地图片文件
		File content1 = new File(courseFile + "/" + fileName);
		content1.delete();//删除本地图片文件

		logger.info("上传成功,FastDFS的地址是:" + fileAbsolutePath);
		/*String replace = fileAbsolutePath.replace("/group1/", "");

		FileInfo group1 = FileManager.getFile("group1", replace);

		Assert.notNull(group1);
		String sourceIpAddr = group1.getSourceIpAddr();
		long size = group1.getFileSize();

		String path = "http://" + sourceIpAddr + fileAbsolutePath;
		logger.info("上传成功,FastDFS的地址是:" + path + "图片大小是:" + size);*/

		return fileAbsolutePath;

	}

	/**
	 * Download.
	 *
	 * @param urlString the url string
	 * @param filename  the filename
	 * @param savePath  the save path
	 * @throws Exception the exception
	 */
	//下载图片到本地
	public static void download(String urlString, String filename, String savePath) throws Exception {

		//判断是否是以http开头不是就加上http://
		if (!urlString.startsWith("http")) {
			urlString = "http://" + urlString;
		}
		// 构造URL
		URL url = new URL(urlString);
		// 打开连接
		URLConnection con = url.openConnection();
		//设置请求超时为5s
		con.setConnectTimeout(5 * 1000);
		// 输入流
		InputStream is = con.getInputStream();

		// 1K的数据缓冲
		byte[] bs = new byte[1024];
		// 读取到的数据长度
		int len;
		// 输出的文件流
		File sf = new File(savePath);
		if (!sf.exists()) {
			sf.mkdirs();
		}
		OutputStream os = new FileOutputStream(sf.getPath() + "/" + filename);
		// 开始读取
		while ((len = is.read(bs)) != -1) {
			os.write(bs, 0, len);
		}
		// 完毕，关闭所有链接
		os.close();
		is.close();
	}

	/*
	* 微信解析首页文章链接
	*
	* */
	public static ArrayList<Map<String, String>> getTitleAndUrl(long time, String substring, HConfig hConfig) {
		JSONObject jsonObject = JSONObject.parseObject(substring);
		JSONArray objects = jsonObject.getJSONArray("list");
		//comm_msg_info //app_msg_ext_info
		ArrayList<Map<String, String>> maps = new ArrayList<>();

		//获取关键字
		String keyword = hConfig.getKeyword();

		for (Object object : objects) {
			Map<String, Object> data = (Map<String, Object>) object;

			Map<String, Object> comm_msg_info = (Map<String, Object>) data.get("comm_msg_info");

			Integer datetime = (Integer) comm_msg_info.get("datetime");

			if (datetime == null) {
				logger.info("时间为空:" + datetime);
				continue;
			}

			long l = datetime.longValue() * 1000;

			if (l < time) {
				logger.info("当前文章时间小于前天的文章时间，不爬取：" + l);
				continue;
			}
			Map<String, Object> app_msg_ext_info = (Map<String, Object>) data.get("app_msg_ext_info");

			List<Map<String, String>> multi_app_msg_item_list = (List<Map<String, String>>) app_msg_ext_info.get("multi_app_msg_item_list");

			if (multi_app_msg_item_list != null && multi_app_msg_item_list.size() > 0) {
				for (Map<String, String> stringStringMap : multi_app_msg_item_list) {
					HashMap<String, String> map1 = new HashMap<>();
					String title1 = stringStringMap.get("title");

					if (StringUtil.isEmpty(title1)) {
						logger.info("当前标题为空跳过爬取:" + title1);
						continue;
					}

					//判断关键字是否为null不为null默认全爬取为null的话只获取带有关键字的title文章
					if (keyword != null && !"".equals(keyword.trim())) {
						if (!title1.contains(keyword)) {
							logger.info("标题不含有关键字，跳过爬取：" + title1);
							continue;
						}
					}


					String content_url1 = stringStringMap.get("content_url");
					String cover1 = stringStringMap.get("cover");

					map1.put("content_url", content_url1);
					map1.put("title", title1);
					map1.put("cover", cover1);
					map1.put("date", String.valueOf(l));
					maps.add(map1);
				}
			}

			HashMap<String, String> map = new HashMap<>();

			String title = (String) app_msg_ext_info.get("title");

			if (StringUtil.isEmpty(title)) {
				logger.info("当前标题为空跳过爬取:" + title);
				continue;
			}

			//判断关键字是否为null不为null默认全爬取为null的话只获取带有关键字的title文章
			if (keyword != null && !"".equals(keyword.trim())) {
				if (!title.contains(keyword)) {
					logger.info("标题不含有关键字，跳过爬取：" + title);
					continue;
				}
			}

			String content_url = (String) app_msg_ext_info.get("content_url");
			String cover = (String) app_msg_ext_info.get("cover");
			map.put("content_url", content_url);
			map.put("title", title);
			map.put("cover", cover);
			map.put("date", String.valueOf(l));
			maps.add(map);
		}
		return maps;
	}

	//重复代码优化
	public static void getMessage(ArrayList<HCrawlLog> hCrawlLogs, HConfig hConfig, String name, DealReptileDao dealReptileDao) {
		try {
			HCrawlLog hCrawlLog = new HCrawlLog();
			hCrawlLog.setStartTime(new Date());//爬虫开始时间

			if ("澎湃新闻".equals(name)) {
				DealReptileServiceImpl.getPaperData(dealReptileDao, hConfig);
			} else if ("36氪".equals(name)) {
				DealReptileServiceImpl.getThirtySixData(dealReptileDao, hConfig);
			} else if ("TEDNEWS".equals(name)) {
				DealReptileServiceImpl.getTedNewsData(dealReptileDao,hConfig);
			} else if ("网易公开课".equals(name)) {
				DealReptileServiceImpl.getWangYiData(dealReptileDao,hConfig);
			} else if ("领英".equals(name)) {
				DealLinkedServiceImpl.getWeChatData(dealReptileDao,hConfig);
			} else if ("爱范儿".equals(name)) {
				DealIfanrServiceImpl.getWeChatData(dealReptileDao,hConfig);
			} else if ("不正常人类研究中心".equals(name)) {
				DealWeChatServiceImpl.getWeChatData(dealReptileDao,hConfig);
			} else if ("糗事百科".equals(name)) {
				DealSpecialReptileServiceImpl.getQiuShiBaiKeData(dealReptileDao,hConfig);
			}

			hCrawlLog.setEndTime(new Date());//爬虫结束时间
			hCrawlLog.setSite(hConfig.getSite());//爬虫网址
			hCrawlLog.setPlatNo(hConfig.getPlatNo());//资讯编号
			hCrawlLog.setKeyword(hConfig.getKeyword());//关键字
			hCrawlLogs.add(hCrawlLog);
		} catch (Exception e) {
			Date date = new Date();
			logger.info("错误!!!===========抓取"+name+"异常========时间是:" + date, e);
		}
	}

	//微信根据网址获取首页html信息
	public static ArrayList<Map<String, String>> getIndexHtml(DealReptileDao dealReptileDao, HConfig hConfig) throws Exception {

		String platNo = hConfig.getPlatNo();
		String weChat = hConfig.getSite();
		String cookie = hConfig.getSecondSite();

		//获取昨天的开始时间
		Calendar cal = new GregorianCalendar();
		cal.setTime(getDayBegin());
		cal.add(Calendar.DAY_OF_MONTH, -1);
		long time = cal.getTime().getTime();

		//根据platNo去资讯表中查询来源
		//判断数据来源是否为空
		if (dealReptileDao.queryMenu(platNo) == null) {
			logger.info("===========没有查询到数据来源==========");
			return null;
		}

		//根据微信号查询网址
		WechatUtil wechatUtil = new WechatUtil();
		wechatUtil.setId(weChat);

		String listUrl = wechatUtil.getListUrl(cookie);

		if (listUrl.equals("") || null == listUrl) {
			logger.info("搜狗微信公众号弹出验证码,需要手动爬取");
			return null;
		}

		Document doc1 = wechatUtil.getDoc(listUrl, cookie);

		String jsonStr = null;

		try {
			jsonStr = doc1.html().split("var msgList = ")[1].split("seajs.use")[0].trim();
		} catch (Exception e) {
			logger.info("搜狗微信公众号弹出验证码,需要手动爬取" + e);
			return null;
		}

		if (jsonStr == null) {
			logger.info("搜狗微信公众号弹出验证码,需要手动爬取 : " + jsonStr);
		}

		String substring = jsonStr.substring(0, jsonStr.length() - 1);


		ArrayList<Map<String, String>> titleAndUrl = getTitleAndUrl(time, substring, hConfig);

		return titleAndUrl;
	}

	//微信获取文章内容并执行插入
	public static void getWeChatMessage(HConfig hConfig, DealReptileDao dealReptileDao, ArrayList<Map<String, String>> indexHtml, String hyzUrl) throws Exception {
		String title;
		String content;
		String author;
		String picture;

		WechatUtil wechatUtil = new WechatUtil();

		for (Map<String, String> stringStringMap : indexHtml) {

			//根据连接获取文章内容,每次请求都暂停3秒
			String content_url = "http://mp.weixin.qq.com" + stringStringMap.get("content_url").replaceAll("amp;", "");
			Document doc = wechatUtil.getDoc(content_url, hConfig.getSecondSite());

			if (null == doc) {
				logger.info("获取文章失败：" + doc);
				continue;
			}

			HMessagePlat hMessagePlat = new HMessagePlat();

			try {
				title = doc.select("#activity-name").first().text();
			} catch (NullPointerException e) {
				logger.info("文章没有title,直接跳过");
				continue;
			}

			//获取文章的时间
			String date = stringStringMap.get("date");

			Long aLong = Long.valueOf(date);
			Date datePublished = new Date(aLong);

			Elements eContent = doc.select("#js_content");
			if (eContent == null && eContent.size() == 0) {
				continue;
			}

			if ("P00013".equals(hConfig.getPlatNo())) {//如果是爱范儿需要去除下面的广告
				//解析数据
				Element first = eContent.first();

				Elements section = first.select("section");
				for (int i = 1; i < section.size(); i++) {
					if (section.get(i).text().contains("后台回复以下关键词获取热文")) {
						for (int j = i; j < section.size(); ) {
							section.remove(j);
						}
						break;
					}
				}
				content = section.toString();
			} else  {
				//解析数据
				content = eContent.first().html();
			}

			//获取文章内容中的content,解析img标签.把src提取出来上传到FastDFS
			Pattern p_img = Pattern.compile("<(img|IMG)(.*?)(/>|></img>|>)");
			Matcher m_img = p_img.matcher(content);

			boolean flag = false;

			boolean result = m_img.find();
			if (result) {
				while (result) {
					String str_img = m_img.group(2);
					//开始匹配<img />标签中的src
					Pattern p_src = Pattern.compile("(src|SRC)=(\"|\')(.*?)(\"|\')");
					Matcher m_src = p_src.matcher(str_img);
					if (m_src.find()) {
						String str_src = m_src.group(3);
						str_src = ReptileUtil.upload(hyzUrl, str_src);//上传图片到FastDFS 返回的地址在替换到内容中
						if (str_src == null) {
							flag = true;
							break;
						}
						str_src = "img前图片标签" + str_src + "img后图片标签";
						content = content.replaceFirst("<(img|IMG)(.*?)(/>|></img>|>)", str_src);
					}
					result = m_img.find();
				}
			}

			//上传fastDFS失败跳过此次循环
			if (flag) {
				continue;
			}

			//将内容中的%替换成全角的%
			Document parse = Jsoup.parse(content);
			String s1 = parse.toString().replaceAll("<p", "p前标签<p")
					.replaceAll("</p>", "p后标签</p>")
					.replaceAll("<section", "前标签<section")
					.replaceAll("</section>", "后标签</section>");
			content = Jsoup.parse(s1).text().replaceAll("p前标签", "<p>")
					.replaceAll("p后标签", "</p>")
					.replaceAll("前标签", "<section>")
					.replaceAll("后标签", "</section>")
					.replaceAll("img前图片标签", "<img src='")
					.replaceAll("img后图片标签", "'/>");


			if (content != null && content.indexOf("%") != -1) {
				content = content.replaceAll("%", "％");
			}

			//获取作者
			author = doc.select("#post-user").first().text();

			Elements select = doc.select("em.rich_media_meta");
			if (select != null && select.size() > 1) {
				author = select.last().text();
			}

			hMessagePlat.setMediaSourceUrl(content_url);
			hMessagePlat.setMessageContent(content);
			hMessagePlat.setMediaSource(dealReptileDao.queryMenu(hConfig.getPlatNo()));
			hMessagePlat.setPublishSub(hConfig.getSite());
			picture = stringStringMap.get("cover");
			picture = ReptileUtil.upload(picture);
			if (picture == null) {
				continue; //如果上传图片失败,直接跳到最外层循环.不要这篇文章了
			}
			hMessagePlat.setMessageImg(picture);

			//将内容中的%替换成全角的%
			if (title != null && title.indexOf("%") != -1) {
				title = title.replaceAll("%", "％");
			}
			hMessagePlat.setMessageTitle(title);
			hMessagePlat.setPublishTime(datePublished);
			hMessagePlat.setAuthor(author);
			hMessagePlat.setPlatNo(hConfig.getPlatNo());
			dealReptileDao.insertWeChatMessage(hMessagePlat);
		}
	}
}
